from email import contentmanager
from re import U
import scrapy
from selenium import webdriver
from wangyiPro.items import WangyiproItem
from selenium.webdriver import ChromeOptions



class WangyiSpider(scrapy.Spider):
    name = 'wangyi'
    # allowed_domains = ['news.163.com']
    start_urls = ['http://news.163.com/']
    models_urls = [] # 存储5个板块对应的url

    # 实例化一个浏览器对象
    def __init__(self, name=None, **kwargs):
        super().__init__(name, **kwargs)
        self.option = ChromeOptions()
        prefs = {
                'profile.default_content_setting_values': {
                    'images': 2
                }
            }
        self.option.headless = True
        self.option.add_experimental_option('prefs', prefs)
        self.option.add_experimental_option('excludeSwitches', ['enable-automation'])
        self.option.add_argument('--ignore-certificate-errors')
        self.bro = webdriver.Chrome(options=self.option)

    # 解析5个板块对应的url
    def parse(self, response):
        li_list = response.xpath('//*[@id="index2016_wrap"]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
        # alist = [3,4,6,7,8]
        alist = [3]
        for index in alist:
            model_url = li_list[index].xpath('./a/@href').extract_first()
            self.models_urls.append(model_url)

        # 依次对每一个模块对应的页面进行请求
        for url in self.models_urls:
            yield scrapy.Request(url,callback=self.parse_model)

    
    def parse_model(self, response, **kwargs):
        # print(response)

        # 由于每一个板块对应的新闻标题相关的内容都是动态加载的，所以以下print无法打印出内容 
        div_list = response.xpath('/html/body/div/div[3]/div[4]/div[1]/div[1]/div/ul/li/div/div')

        for div in div_list:
            title = div.xpath('./div/div/h3/a/text()').extract_first()
            new_detail_href = div.xpath('./div/div/h3/a/@href').extract_first()

            item = WangyiproItem()
            item['title'] = title

            yield scrapy.Request(url=new_detail_href,callback=self.parse_detail,meta={'item':item})


    def parse_detail(self,response):
        content = response.xpath('//*[@id="content"]/div[2]//text()').extract()
        content = ''.join(content)
        item = response.meta['item']
        item['content'] = content

        yield item

    def close(self,spider):
        self.bro.quit()
            

